# -*- coding:utf-8 -*-
# @FileName  :meituan_t1.py
# @Time      :2021/7/6 11:32
import time
from save_mysql_mt import mysql_checking
import random
import re
from math import ceil
from multiprocessing import Pool
import requests
from fake_useragent import UserAgent
from time import sleep
import json
from lxml import html
import sys, io
from selenium import webdriver
from sql_ac.sql_1 import insert_all

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')
sys.setrecursionlimit(1000000)
etree = html.etree
ua = UserAgent()
ck = 0
tunnel = ""
# 用户名密码方式
username = ""
password = ""
proxie = {
    "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel},
    "https": "https://%(user)s:%(pwd)s@%(proxy)s/" % {"user": username, "pwd": password, "proxy": tunnel}
}


def main(ct, key):
    ct1 = str(int(ct.strip().split(":")[2]))

    data = {
        "uuid": "97b411bfe6ec45018e52.1627383806.1.0.0",
        "version": "8.3.3",
        "platform": 3,
        "app": "",
        "partner": 126,
        "riskLevel": 1,
        "optimusCode": 10,
        "originUrl": "http://meishi.meituan.com/i/?ci={}&stid_b=1&cevent=imt%2Fhomepage%2Fcategory1%2F1".format(ct1),
        "offset": 0,
        "limit": 50,
        "cateId": key,  # 类目
        "lineId": 0,
        "stationId": 0,
        "areaId": 0,
        "sort": "default",
        "deal_attr_23": "",
        "deal_attr_24": "",
        "deal_attr_25": "",
        "poi_attr_20043": "",
        "poi_attr_20033": ""
    }
    kv = {
        'user-agent': ua.random,
        'referer': 'https://meishi.meituan.com/i/?cateId=1&onlylist=1',
        'cookie': 'ci=' + ct1
    }
    try:
        r = requests.post('https://meishi.meituan.com/i/api/channel/deal/list', headers=kv, data=data, proxies=proxie, timeout=30).text
    except:
        return main(ct, key)
    req = json.loads(r)
    # input(r)
    try:
        nub = req['data']['poiList']['totalCount']
    except Exception as er:
        print('验证码', r)
        input(er)
        return main(ct, key)
    d = ceil(int(nub) // 50) + 1
    print(f'总页数{d}--城市{ct}--关键词{key}--总数据{nub}')
    # if d < 20:
    #     pool = Pool(d)
    # else:
    #     pool = Pool(20)
    for i in range(0, d + 1):
        print(f'第{i + 1}页--城市{ct}--关键词{key}--总数据{nub}')
        next_1(ct, key, i * 50)
        # pool.apply_async(next_1, (ct, key, i * 50))
    # pool.close()
    # pool.join()


def next_1(ct, key, page):
    ct1 = str(int(ct.strip().split(":")[2]))
    key1 = key.strip().split(":")[1]
    data = {
        "uuid": "97b411bfe6ec45018e52.1627383806.1.0.0",
        "version": "8.3.3",
        "platform": 3,
        "app": "",
        "partner": 126,
        "riskLevel": 1,
        "optimusCode": 10,
        "originUrl": "http://meishi.meituan.com/i/?ci={}&stid_b=1&cevent=imt%2Fhomepage%2Fcategory1%2F1".format(ct1),
        "offset": int(page),
        "limit": 50,
        "cateId": int(key1),  # 类目
        "lineId": 0,
        "stationId": 0,
        "areaId": 0,
        "sort": "default",
        "deal_attr_23": "",
        "deal_attr_24": "",
        "deal_attr_25": "",
        "poi_attr_20043": "",
        "poi_attr_20033": ""
    }
    kv = {
        'user-agent': ua.random,
        'x-requested-with': 'XMLHttpRequest',
        'referer': 'https://meishi.meituan.com/i/?cateId=1&onlylist=1',
        'cookie': 'ci=' + ct1
    }
    try:
        r = requests.post('https://meishi.meituan.com/i/api/channel/deal/list', headers=kv, data=data, proxies=proxie, timeout=30).text
    except:
        return next_1(ct, key, page)
    req = json.loads(r)
    try:
        nub = req['data']['poiList']['totalCount']
    except Exception as er:
        print('验证码1', r)
        input(er)
        return next_1(ct, key, page)
    for i in req['data']['poiList']['poiInfos']:
        # print(i['name'])
        # print(i['poiid'])
        # print(i['ctPoi'])
        print('')
        spider(i['poiid'], i['ctPoi'], ct, key)


def spider(q, w, ct, key):
    ct1 = str(int(ct.strip().split(":")[2]))
    kv1 = {
        'user-agent': ua.random,
        'referer': 'https://meishi.meituan.com/i/?ci={}&stid_b=1&cevent=imt/homepage/category1/1'.format(ct1),
    }
    try:
        r = requests.get('https://meishi.meituan.com/i/poi/{}?ct_poi={}'.format(q, w), headers=kv1, proxies=proxie, timeout=30).text
    except Exception as err1:
        print(err1, '性情错误')
        return spider(q, w, ct, key)
    print('https://meishi.meituan.com/i/poi/{}?ct_poi={}'.format(q, w))
    # print(r)
    try:
        jsdata = json.loads(re.findall(r'(window._appState = )(.*)(;</)', r)[0][1])
    except Exception as err:
        print(re.findall(r'(window._appState = )(.*)(;</)', r), '请求失败-。-', err)
        return spider(q, w, ct, key)
    try:
        phone = jsdata['poiInfo']['phone']
        name = jsdata['poiInfo']['name']
        addr = jsdata['poiInfo']['addr']
    except Exception as wz:
        print(wz, '位置错误')
        return spider(q, w, ct, key)
    print(name + '\n' + phone)
    pp = pe(jsdata['poiInfo']['phone'])
    if pp != '1':
        if mysql_checking(pp):
            try:
                insert_all('meituan_spider',
                           jsdata['poiInfo']['name'].replace("'", "").replace("'", ""), jsdata['poiInfo']['phone'],
                           pp, key.strip().split(":")[0].replace("'", "").replace("'", ""), '',
                           jsdata['poiInfo']['name'].replace("'", "").replace("'", ""), 2,
                           '0', 1, '', '', '', '', '', '', 2, '', '', addr.replace("'", "").replace("'", ""), '', '',
                           'https://meishi.meituan.com/i/poi/{}?ct_poi={}'.format(q, w), '', '', '', 0, '', 0, '', 0, 0, 0, 0, 0,
                           str(time.strftime('%Y-%m-%d %H:%M:%S')), str(time.strftime('%Y-%m-%d %H:%M:%S')), '', 0, 0, '', 0)
                print('入库')
            except:
                pass


def pe(r):
    lu = r.split("/")
    for p in lu:
        if len(p) == 11 and p[0] == '1':
            return p
    return '1'


if __name__ == "__main__":
    with open('cityid11.txt') as fs:
        citys = fs.readlines()
    with open('fodid.txt') as fs:
        keys = fs.readlines()
    '''
    美食:1    0
    火锅:17   1
    聚餐宴请:395    2
    小吃快餐:36   3
    面包甜点:11    4
    饮品店:21329   5
    生日蛋糕:20097    6
    生鲜水果:20638    7
    烧烤:54       8
    '''
    for i in citys[9:10]:
        for j in keys[1:8]:
            # input(i+j)
            main(i, j)
            print('结束一个+112')
        driver = webdriver.Chrome()
        driver.get('https://i.meituan.com/')
